{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('HironsanData/train.txt') as f:\n",
    "    lines = f.readlines()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokens = []\n",
    "labels = []\n",
    "for line in lines:\n",
    "    tokens.append(line.split('\\t')[0].replace('\\n', ''))\n",
    "    labels.append(line.split('\\t')[-1].replace('\\n', ''))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {},
   "outputs": [],
   "source": [
    "# for token, label in zip(tokens, labels):\n",
    "#     print(token, label)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "metadata": {},
   "outputs": [],
   "source": [
    "org_labels = ['I-ORG', 'B-DAT', 'B-ORG', 'I-ART', 'I-MNY', 'B-TIM', 'I-PNT', 'I-PSN', 'I-DAT','B-PNT', 'B-PSN',\n",
    "                'I-LOC', 'B-MNY', 'B-LOC', 'B-ART', 'I-TIM']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {},
   "outputs": [],
   "source": [
    "convert_labels = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i, label in enumerate(labels):\n",
    "    if label == '' or tokens[i] == '':\n",
    "        convert_labels.append('')\n",
    "    elif label in org_labels:\n",
    "        convert_labels.append('O')\n",
    "    else:\n",
    "        convert_labels.append('X')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {},
   "outputs": [],
   "source": [
    "output_lines = []\n",
    "for token, label in zip(tokens, convert_labels):\n",
    "    output_lines.append(token + '\\t' + label)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('KeywordData/train.txt', 'w') as f:\n",
    "    f.write('\\n'.join(output_lines))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
